import pickle,re
from tqdm import tqdm
'''
[
'荆毅定居城市的第一件事，是给三岁的女儿找幼儿园，妻命我去寻。', 
[[0, 1], [2, 3], [4, 5], [6], [7, 8], [9], [10], [11], [12], [13], [14], [15], [16], [17, 18], [19], [20, 21, 22], [23], [24], [25], [26], [27], [28], [29]], 
[(8, 'sbj_HX', 6), (8, 'obj_HX', 14), (14, '_BY', 13), (14, 'obj_HX', 15), (18, 'sbj_HX', 17), (18, 'obj_HX', 19), (21, 'sbj_HX', 19)], 
[('是', 'sbj_HX', '事'), ('是', 'obj_HX', '找'), ('找', '_BY', '女儿'), ('找', 'obj_HX', '幼儿园'), ('命', 'sbj_HX', '妻'), ('命', 'obj_HX', '我'), ('寻', 'sbj_HX', '我')], 
['荆毅/nr', '定居/v', '城市/n', '的/u', '第一/m', '件/q', '事/n', '，/w', '是/v', '给/p', '三/m', '岁/q', '的/u', '女儿/n', '找/v', '幼儿园/n', '，/w', '妻/Ng', '命/n', '我/r', '去/v', '寻/v', '。/w']
]
'''
def manual():
    c = 0
    result_sent, sent, record, idx, cixing = [], [], [], [], []
    with open('gold_manul_1000.txt','r',encoding='utf-8',errors='ignore') as file:
        content = file.readlines()
        for line in tqdm(content):
            ret = re.search('@(.+)',line.strip())
            if ret:
                flag = 0
                sentence = ret.group(1)
                pos = []
                for k in sentence.split():
                    pos.append(k.split('_')[1])
                fenci = []
                for k in re.sub('\/[a-zA-Z]*','',sentence).split(' '):
                    fenci.append(k.split('_')[1])
                sent.append(''.join(i for i in fenci))
                num,i=[],0   
                for k in fenci:
                    num.append(list(range(i,i+len(k))))
                    i=i+len(k) 
            elif line.strip() == "====PGraph Beg====":
                continue
            elif line.strip() == "====PGraph End====":
                if flag == 0:
                    sent.append(num)
                    sent.append(idx)
                    sent.append(record)
                    sent.append(pos)
                    result_sent.append(sent)
                else:
                    flag = 0
                sent, record, idx, cixing = [], [], [], []
            elif len(line.strip()) != 0:
                if '|' not in line.strip():
                    head = line.strip().split()
                    record.append((head[0].split('_')[1],head[1],head[2],head[3].split('_')[1]))
                    x = int(head[0].split('_')[0])
                    y = int(head[3].split('_')[0])
                    idx.append((x,head[1],head[2],y))
                elif '|d' in line.strip():
                    # c += 1
                    continue
                elif '|p' in line.strip():
                    # c += 1
                    head = line.strip().split()
                    record.append((head[0].split('_')[1],head[1],head[2],head[3].split('_')[1]))
                    x = int(head[0].split('_')[0])
                    y = int(head[3].split('_')[0])
                    idx.append((x,head[1],head[2],y))
                else:  # 修改的
                    c += 1
                    new_head = line.strip().split('|')[1]
                    head = new_head.strip().split()
                    # print(line)
                    record.append((head[0].split('_')[1],head[1],head[2],head[3].split('_')[1]))
                    x = int(head[0].split('_')[0])
                    y = int(head[3].split('_')[0])
                    idx.append((x,head[1],head[2],y))
    
    return result_sent , c

if __name__=="__main__":
    final_data = []
    with open('final_data.pkl', 'rb') as out_file:
        final_data = pickle.load(out_file)
    print(len(final_data),final_data[0])  # 212784

    list_info = [25, 48, 326, 367, 425, 443, 518, 538, 630, 799, 951, 1008, 1100, 1273, 1288, 1318, 1391, 1408, 1574, 1718, 1768, 1838, 1896, 1947, 1951, 1999, 2394, 2456, 2510, 2683, 2788, 2809, 2871, 3034, 3043, 3123, 3126, 3262, 3334, 3463, 3575, 3714, 3756, 3783, 3825, 3845, 3852, 3904, 4029, 4057, 4207, 4240, 4303, 4330, 4357, 4572, 4733, 4824, 4884, 4959, 5044, 5086, 5104, 5171, 5421, 5447, 5511, 5548, 5583, 5609, 5862, 6367, 6412, 6483, 6600, 6641, 6657, 6730, 6748, 6824, 6942, 6947, 6982, 7074, 7151, 7164, 7167, 7307, 7365, 7414, 7470, 7495, 7511, 7642, 7767, 7770, 8113, 8279, 8574, 8576, 8799, 8905, 8936, 8979, 8989, 9099, 9152, 9250, 9308, 9326, 9367, 9400, 9488, 9506, 9701, 9751, 9762, 9776, 9813, 9883, 9896, 9956, 9993, 10101, 10221, 10227, 10303, 10378, 10441, 10452, 10473, 10546, 10591, 10612, 10759, 10775, 10820, 10999, 11067, 11069, 11093, 11138, 11144, 11215, 11363, 11371, 11405, 11424, 11485, 11504, 11970, 12035, 12178, 12283, 12562, 12632, 12736, 12738, 12835, 12890, 12951, 13152, 13158, 13167, 13389, 13454, 13554, 13563, 13896, 13978, 14100, 14145, 14156, 14187, 14238, 14252, 14283, 14325, 14555, 14692, 14770, 14792, 14806, 15175, 15449, 15593, 15637, 15689, 15720, 15840, 15867, 15917, 16000, 16051, 16069, 16412, 16482, 16600, 16603, 16705, 16844, 16925, 16928, 16937, 16938, 16958, 17075, 17191, 17196, 17319, 17381, 17481, 17516, 17642, 17728, 17855, 17857, 17988, 18052, 18059, 18092, 18200, 18438, 18537, 18589, 18699, 18740, 18891, 18991, 19006, 19062, 19161, 19222, 19305, 19394, 19501, 19683, 19794, 20053, 20310, 20500, 20528, 20555, 20654, 20732, 20784, 21001, 21028, 21251, 21296, 21319, 21344, 21549, 21870, 22124, 22178, 22301, 22449, 22461, 22641, 22660, 22900, 22992, 23006, 23032, 23188, 23215, 23311, 23491, 23551, 23601, 23659, 23834, 23838, 23946, 23992, 24046, 24144, 24195, 24258, 24325, 24417, 24540, 24557, 24680, 24957, 25039, 25153, 25225, 25543, 25566, 25648, 25660, 25706, 25723, 25808, 25811, 25881, 25943, 25953, 25991, 26042, 26280, 26472, 26503, 26596, 26750, 26867, 26903, 26976, 27025, 27033, 27054, 27062, 27074, 27164, 27212, 27305, 27326, 27419, 27575, 27604, 27626, 27633, 27673, 27707, 27800, 27844, 27866, 27879, 28047, 28059, 28061, 28117, 28170, 28196, 28247, 28248, 28305, 28400, 28427, 28460, 28541, 28550, 28629, 28743, 28909, 28959, 29054, 29070, 29082, 29087, 29109, 29260, 29360, 29370, 29373, 29394, 29427, 29493, 29645, 29680, 29738, 29764, 29908, 29974, 30021, 30042, 30049, 30158, 30178, 30229, 30411, 30465, 30488, 30656, 30733, 30969, 30974, 30985, 30992, 31049, 31094, 31267, 31353, 31487, 31521, 31740, 31747, 31790, 31854, 31897, 32137, 32183, 32326, 32348, 32402, 32415, 32502, 32659, 32678, 32705, 32709, 32762, 32832, 33136, 33197, 33232, 33403, 33521, 33553, 33757, 33787, 33978, 33996, 33998, 34130, 34204, 34231, 34237, 34284, 34303, 34453, 34639, 34648, 34750, 34753, 34764, 34820, 34929, 35197, 35201, 35257, 35346, 35457, 35550, 35903, 35978, 36217, 36291, 36384, 36407, 36504, 36512, 36722, 36816, 36859, 36954, 36996, 36998, 37288, 37339, 37359, 37390, 37392, 37466, 37491, 37557, 37701, 37746, 37790, 37935, 38014, 38114, 38171, 38196, 38455, 38489, 38702, 38705, 38900, 38958, 38971, 38981, 38990, 39056, 39062, 39203, 39261, 39354, 39381, 39412, 39522, 39594, 39651, 39879, 40026, 40030, 40035, 40245, 40263, 40298, 40338, 40353, 40491, 40505, 40514, 40565, 40799, 40822, 40845, 40858, 40944, 40997, 41221, 41488, 41567, 41596, 41799, 41862, 42008, 42022, 42202, 42306, 42564, 42822, 43226, 43355, 43366, 43414, 43464, 43545, 43560, 43576, 43674, 43805, 43928, 43997, 44085, 44267, 44270, 44310, 44330, 44446, 44768, 44789, 44819, 44927, 45125, 45142, 45189, 45211, 45216, 45335, 45464, 45484, 45537, 45597, 45662, 45672, 45676, 45950, 45969, 46079, 46243, 46323, 46391, 46730, 46804, 47004, 47038, 47280, 47423, 47539, 47572, 47720, 47892, 47925, 48322, 48402, 48525, 48618, 48647, 48664, 48729, 48787, 48983, 49059, 49219, 49255, 49339, 49439, 49593, 49732, 49754, 49811, 49952, 50009, 50133, 50148, 50152, 50220, 50287, 50359, 50405, 50479, 50507, 50508, 50570, 50721, 50789, 50955, 51021, 51064, 51240, 51246, 51340, 51401, 51594, 51614, 51740, 51741, 51816, 51892, 51895, 51910, 51926, 51936, 51965, 52070, 52150, 52294, 52369, 52533, 52694, 52759, 52864, 52981, 53298, 53407, 53533, 53600, 53647, 53845, 54008, 54036, 54097, 54320, 54416, 54491, 54513, 54614, 54633, 54963, 55008, 55014, 55036, 55150, 55186, 55217, 55335, 55404, 55405, 55475, 55573, 55605, 55662, 55667, 55861, 55865, 55867, 55869, 55966, 56007, 56033, 56045, 56123, 56414, 56449, 56606, 56675, 56920, 57147, 57272, 57278, 57344, 57352, 57468, 57525, 57898, 58026, 58318, 58324, 58433, 58757, 58842, 58865, 58884, 58910, 59256, 59281, 59339, 59425, 59434, 59455, 59501, 59584, 59653, 59656, 59724, 59755, 59792, 59806, 59979, 60078, 60197, 60414, 60657, 60701, 60723, 60847, 60852, 60957, 61056, 61103, 61172, 61210, 61220, 61305, 61347, 61349, 61356, 61383, 61605, 61620, 61868, 61890, 61897, 62024, 62078, 62160, 62210, 62213, 62465, 62603, 62625, 62736, 62742, 62762, 62830, 62884, 62906, 63043, 63200, 63214, 63227, 63251, 63259, 63299, 63409, 63588, 63886, 63902, 63942, 63945, 63947, 64102, 64170, 64174, 64212, 64334, 64383, 64486, 64531, 64658, 64736, 64780, 64834, 64951, 64982, 65333, 65382, 65453, 65510, 65524, 65656, 65841, 65923, 65970, 66014, 66034, 66313, 66608, 66986, 66992, 67025, 67036, 67096, 67125, 67302, 67382, 67461, 67559, 67577, 67665, 67797, 67804, 67807, 68000, 68074, 68084, 68091, 68154, 68226, 68241, 68625, 68630, 68796, 68818, 68914, 68951, 69290, 69478, 69606, 69689, 69705, 69732, 69838, 69920, 70164, 70224, 70327, 70399, 70420, 70503, 70619, 70623, 70853, 70858, 70983, 71041, 71044, 71116, 71174, 71228, 71361, 71371, 71547, 71623, 71721, 71726, 71789, 71851, 71856, 71935, 72044, 72194, 72198, 72237, 72288, 72289, 72352, 72387, 72428, 72584, 72605, 72630, 72637, 72814, 72860, 72870, 73223, 73263, 73276, 73401, 73514, 73568, 73609, 73718, 73759, 73763, 73775, 73900, 73975, 74153, 74198, 74424, 74439, 74630, 74654, 74765, 74789, 74883, 74929, 75198, 75306, 75354, 75686, 75741, 75755, 75767, 75819, 75885, 75916, 76075, 76086, 76107, 76289, 76318, 76439, 76472, 76511, 76557, 76558, 76583, 76647, 76825, 76961, 77140, 77192, 77196, 77343, 77415, 77465, 77480, 77550, 77555, 77730, 77750, 77805, 77862, 77901, 77935, 78119, 78151, 78240, 78442, 78451, 78485, 78540, 78605, 78710, 78775, 78918, 78953, 78998, 79038, 79176, 79208, 79357, 79399, 79488, 79620, 79638, 79772, 79796, 79830, 79893, 79929, 80115, 80171, 80194, 80370, 80409, 80441, 80497, 80532, 80542, 80594, 80610, 80640, 80840, 80870, 80889, 80917, 80976, 81073, 81171, 81184, 81251, 81320, 81474, 81648, 81657, 81682, 81691, 81741, 81853, 82032, 82033, 82048, 82079, 82183, 82191, 82274, 82312, 82432, 82454, 82512, 82587, 82658, 82669, 82728, 82801, 82887, 82933, 82945, 82947, 83030, 83118]
    gold_data, train_val = [], []
    for i in range(len(final_data)):
        if i in list_info:
            gold_data.append(final_data[i])
        else:
            train_val.append(final_data[i])
    print(len(gold_data),len(train_val))  # 1000, 82182
    new_gold = []
    for item in gold_data:
        N, po, t = [], [], []
        for i in range(len(item[-1])):
            po.append(str(i) + '_' + item[-1][i])
        N.append(po)
        for a,b in zip(item[2],item[3]):
            t.append((str(a[0]) + '_' + b[0], a[1], a[2], str(a[3]) + '_' + b[3]))
        N.append(t)
        new_gold.append(N)
    with open('gold_data.txt','w',encoding = 'utf-8') as file:
        for item in new_gold:
            file.write('@' + ' '.join(i for i in item[0]) + '\n')
            file.write("====PGraph Beg====\n")
            for key in item[1]:  # ('是', 'sbj_HX', 'Exp', '事')
                file.write(key[0] + ' ' + key[1] + ' ' + key[2] + ' ' + key[3] + '\n')
            file.write("====PGraph End====\n\n\n")

    manual_data, c = manual()
    print("len(result_sent):", len(manual_data), manual_data[0])
    print(gold_data[0])
    print(manual_data[-1])
    print(gold_data[999])
    count = 0
    for a in gold_data[:1000]:
        count += len(a[2])
    print(c, count, count - c, (count - c)/count)
    with open('gold_manual_data1000.pkl','wb') as out_file:
        pickle.dump(manual_data,out_file)
   
                    


            